package com.mao.Spider;


import com.fasterxml.jackson.databind.ObjectMapper;
import com.mao.dao.LinuxDao;
import com.mao.pojo.LinuxPack;
import com.mao.utils.MyBatisUtils;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

/**
 * @Description:
 * @Param:
 * @Thought:
 * @Author: Wenjie Mao
 * @create： 2022-12-06 15:35
 */
public class LinuxSpider {
    //等待爬取的url
    private static volatile Queue<String> queue = new LinkedBlockingQueue<String>();
    //爬取过的url
    private static volatile Set<String> allOverUrl=new HashSet<>();
    //总线程5条
    private static int MAX_THREAD=5;
    //开始时间
    private static long startTime=0;
    //记录获取的邮箱个数
    private static int emailCount=0;

    public static void main(String[] args) {
        LinuxSpider linuxSpider = new LinuxSpider();
        for (int i = 1; i <= 138; i++) {
            System.out.println("===================================Page " + i + "==============================");
            linuxSpider.indexPage(i);
        }

    }

    public LinuxPack dealPackage(String url){
        Connection conn = null;
        Document document = null;

        LinuxPack linuxPack = new LinuxPack();


        try {

            conn = Jsoup.connect(url).ignoreContentType(true);

            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            document = conn.get();

            Elements nameTag = document.getElementsByAttributeValue("itemprop", "name");
            Elements versionTag = document.getElementsByAttributeValue("itemprop", "version");

            String name = nameTag.attr("content");
            String version = versionTag.attr("content");
            linuxPack.setName(name);
            linuxPack.setVersion(version);

            Element mainBody = document.getElementById("pkginfo");
            Elements th = mainBody.getElementsByTag("th");
            for (Element element : th) {
                String tmpText = element.text();
                String[] split = tmpText.split(":");
                String textCase = split[0];
                Element element1 = element.nextElementSibling();
                String text = element1.text();

                //设置pojo的值
                switch (textCase){
                    case "Architecture":
                        Elements a = element1.getElementsByTag("a");
                        text = a.get(0).text();
                        linuxPack.setArchitecture(text);
                        break;
                    case "Repository":
                        Elements a1 = element1.getElementsByTag("a");
                        text = a1.get(0).text();
                        linuxPack.setRepository(text);
                        break;
                    case "Description":
                        text = element1.text();
                        linuxPack.setDescription(text);
                        break;
                    case "License(s)":
                        text = element1.text();
                        linuxPack.setLicense(text);
                        break;
                    case "Package Size":
                        text = element1.text();
                        String[] mbs = text.split(" ");
                        text = mbs[0];
                        linuxPack.setPackageSize(Double.valueOf(text));
                        break;
                    case "Installed Size":
                        text = element1.text();
                        String[] mbs2 = text.split(" ");
                        text = mbs2[0];
                        linuxPack.setInstalledSize(Double.valueOf(text));
                        break;
                    case "Build Date":
                        text = element1.text();
                        linuxPack.setBuildDate(text);
                        break;
                    case "Signature Date":
                        text = element1.text();
                        linuxPack.setSignatureDate(text);
                        break;
                    case "Last Updated":
                        text = element1.text();
                        linuxPack.setLastUpdated(text);
                        break;

                }
            }

            ArrayList<String> dependList = new ArrayList<>();
            ArrayList<String> requireList = new ArrayList<>();

            Element dependencies = document.getElementById("pkgdepslist");

            if(dependencies != null){
                Elements dli = dependencies.getElementsByTag("li");

                for (Element element : dli) {
                    Elements a = element.getElementsByTag("a");
                    Element a1 = a.get(0);
                    dependList.add(a1.text());
                }
            }

            Element requires = document.getElementById("pkgreqslist");

            if(requires != null){
                Elements rli = requires.getElementsByTag("li");

                for (Element element : rli) {
                    Elements a = element.getElementsByTag("a");
                    Element a1 = a.get(0);
                    requireList.add(a1.text());
                }
            }

            linuxPack.setDependencies(dependList.toString().replace("[","").replace("]",""));
            linuxPack.setRequiredBy(requireList.toString().replace("[","").replace("]",""));

        }catch (Exception e){
            e.printStackTrace();
        }
        return linuxPack;
    }

    /**
     *
     */
    public void indexPage(int pageIndex){
        String url = "https://archlinux.org/packages/?page="+pageIndex+"&";
        Connection conn = null;
        Document document = null;
        ObjectMapper mapper = new ObjectMapper();
        SqlSession sqlSession = MyBatisUtils.getSqlSession();
        LinuxDao linuxDao = sqlSession.getMapper(LinuxDao.class);
        try {

            conn = Jsoup.connect(url).ignoreContentType(true);

            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            document = conn.get();

            Elements elementsByTag = document.getElementsByTag("td");

            for (Element element : elementsByTag) {
                Elements a = element.getElementsByTag("a");
                if(!a.isEmpty()){
                    String text = a.text();
                    String href = a.attr("href");

                    LinuxPack linuxPack = dealPackage("https://archlinux.org" + href);

                    linuxDao.insert(linuxPack);

                    System.out.println("inserting\t" + linuxPack.getName());
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {

        }

    }



    /**
     * @url https://archlinux.org/packages/?page=1&
     * @param pageIndex
     */
    public void webToJson(int pageIndex){
        Connection conn = null;
        Document document = null;
        FileOutputStream fileOutputStream = null;
        try {
            String url = "https://gitee.com/api/v5/search/repositories?q=Spark&page="+pageIndex+"&per_page=100&order=desc";
            conn = Jsoup.connect(url).ignoreContentType(true);

            conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.header("Accept-Encoding", "gzip, deflate, sdch");
            conn.header("Accept-Language", "zh-CN,zh;q=0.8");
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
//            document = Jsoup.connect(url).header("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2").ignoreContentType(true).get();

            document = conn.get();

            Elements elements = document.getElementsByTag("body");
            Element element = elements.get(0);
            //获取到网页的源代码
            String s = element.toString();


        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                fileOutputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

